package com.kaowen.spider;

import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.ArrayListHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.helper.DataUtil;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import com.google.gson.Gson;
import com.kaowen.spider.util.DBManager;

/**
 * 网络爬虫 ipin网公司数据
 * 
 * @author thinkpad
 */
public class Spider {

	private static Logger logger = Logger.getLogger(Spider.class);

	private Connection conn = null;

	private Gson gson = new Gson();

	private String savePath = "D:\\ipin\\";

	public Spider() {
		try {
			conn = DBManager.getConnection();
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}

	/**
	 * 初始化用户信息，用于抓取操作
	 */
	public void initUser() {
		QueryRunner queryRunner = new QueryRunner();
		try {
			queryRunner.update(conn, "update sp_user t set t.state = 0 ,count = 0");
		} catch (SQLException e) {
			e.printStackTrace();
		}
	}

	/**
	 * 获取一个IpinUserId
	 * 
	 * @param initUser
	 *            是否初始化用户
	 * @return
	 */
	private String getIpinUserId(boolean initUser) {
		if (initUser)
			initUser();
		QueryRunner queryRunner = new QueryRunner();
		String ipinUser = null;
		try {
			List<Object[]> ipinUserIdList = queryRunner
					.query(conn,
							"select t.ipinuserid from sp_user t where t.state = 0 and t.ipinuserid is not null and rownum <= 1",
							new ArrayListHandler());
			for (Object[] ipinUserId : ipinUserIdList) {
				ipinUser = (String) ipinUserId[0];
			}
		} catch (SQLException e) {
			e.printStackTrace();
		}
		try {
			queryRunner.update(conn, "update sp_user t set t.state = 1 where t.ipinuserid = ?", ipinUser); // 更新用户状态为1使用中
		} catch (SQLException e) {
			e.printStackTrace();
		}
		logger.info("获得一个ipinUserId" + ipinUser + "！");
		if (ipinUser == null)
			noUser = true;
		return ipinUser;
	}

	/**
	 * 抓取公司列表
	 */
	public void capCompany() {
		int num = 0;
		QueryRunner queryRunner = new QueryRunner();
		String sql = "INSERT INTO SP_COMPANY (SP_COMPANY_ID,COMPANY_NAME,INDUSTRY,URL,INSERT_TIME,UPDATE_TIME) VALUES (?,?,?,?,sysdate,sysdate)";
		while (true) {
			String url = "http://www.ipin.com/search/searchResult.do?searchKey=&&num=9999%2B&pageNum=1&searchType=inc&&pageMode=1&len=10&pipeId=pipe_2_2&jsUid=2&_=1417069968982";
			try {
				String rsStr = Jsoup.connect(url).data("page", String.valueOf(num)).execute().body();
				if (rsStr.contains("未检索到相关公司"))
					break;
				int begin = rsStr.indexOf("pipe.fire(\"search/result\",{\"html\": \"\\n");
				int end = rsStr.indexOf("\\n\", \"json\": {\"searchKey\": \"\", \"loc\":");
				rsStr = rsStr.substring(begin + "pipe.fire(\"search/result\",{\"html\": \"\\n".length(), end);
				rsStr = rsStr.replace("\\n", "");
				rsStr = rsStr.replace("\\\"", "\"");
				ByteArrayInputStream is = new ByteArrayInputStream(rsStr.getBytes());
				Document doc = DataUtil.load(is, "UTF-8", "http://www.ipin.com/search/searchResult.do");
				Elements elesCompany = doc.select(".company-name");
				Elements elesInd = doc.select(".ind");
				for (int i = 0; i < elesCompany.size(); i++) {
					String companyName = elesCompany.get(i).text();
					String companyUrl = elesCompany.get(i).attr("href");
					String industry = elesInd.get(i).text();
					queryRunner.update(conn, sql, companyUrl, companyName, industry, companyUrl);
				}
			} catch (Exception e) {
				try {
					queryRunner.update(conn, "insert into SP_ERROR_LOG (LOG_TIME,SUBJECT,MSG) values (sysdate,?,?)",
							"capSchool", e.toString());
				} catch (SQLException e1) {
				}
				e.printStackTrace();
			}
			num++;
		}
	}

	/**
	 * 抓取公司详情
	 */
	public void capCompanyDetail() {
		long beginTime = System.currentTimeMillis();
		QueryRunner queryRunner = new QueryRunner();
		String querySql = "select t.sp_company_id from sp_company t where t.status = 0 and rownum <= 50";
		String successSql = "update sp_company t set t.status = ? , t.finish_time = sysdate , t.file_size = ? where t.sp_company_id = ?";
		String errorLogSql = "insert into SP_ERROR_LOG (LOG_TIME,SUBJECT,MSG) values (sysdate,?,?)";
		while (true) {
			try {
				// 寻找状态为0的可用用户
				List<Object[]> ipinUserIdList = queryRunner
						.query(conn,
								"select t.ipinuserid from sp_user t where t.state = 0 and t.ipinuserid is not null and rownum <= 1",
								new ArrayListHandler());
				if (ipinUserIdList.size() > 0) { // 开始抓取
					String ipinUserId = (String) ipinUserIdList.get(0)[0];
					logger.info("开始使用" + ipinUserId + "进行抓取！");
					queryRunner.update(conn, "update sp_user t set t.state = 1 where t.ipinuserid = ?", ipinUserId);// 更新用户状态为1使用中
					while (true) {
						try {
							List<Object[]> keyArray = queryRunner.query(conn, querySql, new ArrayListHandler()); // 寻找状态为0的公司id，每次取50个
							if (keyArray.size() > 0) {
								boolean breakFlag = false;
								for (Object[] keyA : keyArray) {
									String url = "http://www.ipin.com/company/profile/i_" + keyA[0] + "/";
									try {
										String rsStr = Jsoup.connect(url).cookie("ipinUserId", ipinUserId).execute()
												.body();
										if (rsStr.contains("您的访问太频繁，请明天再来")) {
											logger.info("账号" + ipinUserId + "今日配额使用完毕，更换下一个账号抓取！");
											breakFlag = true;
										} else {
											if (rsStr.length() > 2000) {
												BufferedWriter buf = new BufferedWriter(new FileWriter(savePath
														+ keyA[0] + ".html")); // 写入文件
												buf.write(rsStr);
												buf.close();
												queryRunner.update(conn, successSql, 9, rsStr.length(), keyA[0]); // 更新公司状态为9已完成
												logger.info("抓取" + keyA[0] + "成功，已保存至" + savePath + keyA[0] + ".html");
											} else {
												queryRunner.update(conn, successSql, -1, rsStr.length(), keyA[0]); // 更新公司状态为-1
																													// 有问题
												logger.error("抓取" + keyA[0] + "失败，抓取文件过小，未保存!");
											}
										}
									} catch (IOException e) {
										e.printStackTrace();
										queryRunner.update(conn, successSql, -1, 0, keyA[0]);
										queryRunner.update(conn, errorLogSql, "capCompanyDetail", e.toString());
									}
									queryRunner.update(conn,
											"update sp_user t set t.count = t.count+1  where t.ipinuserid = ?",
											ipinUserId);// 用户抓取计数器
								}
								if (breakFlag)
									break;
							} else {
								break;
							}
						} catch (Exception e1) {
							e1.printStackTrace();
						}
					}
					queryRunner.update(conn, "update sp_user t set t.state = 9 where t.ipinuserid = ?", ipinUserId); // 更新用户状态为9禁用
				} else {
					logger.info("账号资源用尽，请明日继续抓取");
					break;
				}
			} catch (Exception e) {
				logger.error("查寻账号资源时出现异常：" + e.toString());
			}
		}
		logger.info("抓取完成 执行耗时 : " + (System.currentTimeMillis() - beginTime) / 1000f + " 秒 ");
	}

	/**
	 * 解析公司详细信息
	 */
	public void analysisCompanyDetail() {
		QueryRunner queryRunner = new QueryRunner();
		String querySql = "select t.sp_company_id from sp_company t where t.status = 9 and rownum <= 10";
		try {
			List<Object[]> companyIdList = queryRunner.query(conn, querySql, new ArrayListHandler());
			for (Object[] companyIdObject : companyIdList) {
				String companyId = (String) companyIdObject[0];
				Document doc = DataUtil.load(new File(savePath + companyId + ".html"), "UTF-8", "www.ipin.com");
				Map<String, Object> bean = new TreeMap<String, Object>();

				// 企业概况
				bean.put("公司名称company_name", doc.select(".company .name").text());
				bean.put("公司类型", doc.select(".trade h3").text());
				bean.put("雇主评级", doc.select(".trade .score").text());
				bean.put("薪酬指数", (doc.select(".appraisal ul li").get(0).select("span").get(0).text().replace("％", ""))
						.replace("%", ""));
				bean.put("稳定指数", (doc.select(".appraisal ul li").get(1).select("span").get(0).text().replace("％", ""))
						.replace("%", ""));
				bean.put("公司规模", (doc.select(".appraisal ul li").get(2).select("span").get(0).text().replace("％", ""))
						.replace("%", ""));
				bean.put("样本描述", doc.select(".appraisal .sample-txt").text());
				bean.put("公司简介", (doc.select(".des-all").text().replace("公司简介：", "")).replace("收起", ""));

				String docString = null;
				String findLabel = null;
				int begin = 0;
				int end = 0;
				Map gsonMap = null;

				// 员工概况
				docString = doc.html();
				findLabel = "<script>pipe.fire(\"similar/similar\",";
				begin = docString.indexOf(findLabel);
				end = docString.indexOf(");</script>", begin);
				docString = docString.substring(begin + findLabel.length(), end);
				gsonMap = gson.fromJson(docString, Map.class);
				for (Object cvMapObj : ((List) ((Map) gsonMap.get("json")).get("cvList"))) {
					Map cvMap = (Map) cvMapObj;
					bean.put("_id", cvMap.get("_id"));
					bean.put("cvId", cvMap.get("cvId"));
					bean.put("age", cvMap.get("age"));
					bean.put("gender", cvMap.get("gender"));
					bean.put("gender", cvMap.get("gender"));
					bean.put("diploma", cvMap.get("diploma"));
					bean.put("eduStart", cvMap.get("eduStart"));
					bean.put("eduEnd", cvMap.get("eduEnd"));
					bean.put("eduSch", cvMap.get("eduSch"));
					bean.put("eduSchId", cvMap.get("eduSchId"));
					bean.put("eduMajor", cvMap.get("eduMajor"));
					bean.put("eduMajorId", cvMap.get("eduMajorId"));
					bean.put("jobStart", cvMap.get("jobStart"));
					bean.put("jobInc", cvMap.get("jobInc"));
					bean.put("jobIncId", cvMap.get("jobIncId"));
					bean.put("jobInd", cvMap.get("jobInd"));
					bean.put("jobLoc", cvMap.get("jobLoc"));
					bean.put("jobPosition", cvMap.get("jobPosition"));
					bean.put("jobCate", cvMap.get("jobCate"));
					bean.put("jobSalary", cvMap.get("jobSalary"));
					bean.put("jobIndex", cvMap.get("jobIndex"));
					bean.put("inc_rank", cvMap.get("inc_rank"));
					bean.put("inc_rank_width", cvMap.get("inc_rank_width"));
					bean.put("sch_rank", cvMap.get("sch_rank"));
					bean.put("sch_rank_width", cvMap.get("sch_rank_width"));
					bean.put("tag", cvMap.get("tag"));
					bean.put("showIncName", cvMap.get("showIncName"));
				}

				// 招怎样的人
				// 1)学历
				docString = doc.html();
				findLabel = "<script>pipe.fire(\"company/details/edu\",";
				begin = docString.indexOf(findLabel);
				end = docString.indexOf(");</script>", begin);
				docString = docString.substring(begin + findLabel.length(), end);
				gsonMap = gson.fromJson(docString, Map.class);
				for (Object cvMapObj : ((List) ((Map) gsonMap.get("json")).get("diploma"))) {
					Map cvMap = (Map) cvMapObj;
					bean.put("pre", cvMap.get("pre"));
					bean.put("indPre", cvMap.get("indPre"));
					bean.put("name", cvMap.get("name"));
				}

				// 2)性别
				bean.put("男女比例 男", doc.select(".gender .box .left").text());
				bean.put("男女比例 女", doc.select(".gender .box .right").text());
				bean.put("行业平均比例", doc.select(".gender .sub-txt").text());

				for (String key : bean.keySet()) {
					System.out.println(key + "：\t" + bean.get(key));
				}
				System.out.println("-------------------------------------------------------------------");
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	List<Thread> threadList = new LinkedList<Thread>();

	private boolean noUser = false; // 标记是否有用户

	/**
	 * 多线程抓取公司信息
	 * 
	 * @param threadCount
	 *            线程数量
	 */
	public void capCvListMultiThread(int threadCount) {
		QueryRunner queryRunner = new QueryRunner();
		String querySql = "select t.sp_company_id from sp_company t where (t.cv_status = 0 or t.cv_status is null) and rownum <= 1";
		while (true) {
			if (noUser)
				break;
			Iterator<Thread> iter = threadList.iterator();
			while (iter.hasNext()) {
				Thread thread = iter.next();
				if (!thread.isAlive())
					iter.remove();
			}
			if (threadList.size() < threadCount) {
				try {
					List<Object[]> keyArray = queryRunner.query(conn, querySql, new ArrayListHandler());
					if (keyArray.size() > 0) {
						Thread thread = new Thread(new CapCvTask((String) keyArray.get(0)[0]));
						thread.start();
						threadList.add(thread);
					} else {
						continue;
					}
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
			try {
				Thread.sleep(1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		logger.info("所有公司已抓取完毕");
	}

	/**
	 * 单线程
	 */
	public void capCvList() {
		String querySql = "select t.sp_company_id from sp_company t where (t.cv_status = 0 or t.cv_status is null) and rownum <= 50";
		QueryRunner queryRunner = new QueryRunner();
		while (true) {
			try {
				List<Object[]> keyArray;
				keyArray = queryRunner.query(conn, querySql, new ArrayListHandler());
				if (keyArray.size() > 0) {
					for (Object[] keyA : keyArray) {
						CapCvTask task = new CapCvTask((String) keyA[0]);
						task.run();
					}
				} else {
					break;
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}

	/**
	 * 抓取CvList线程
	 * 
	 * @author thinkpad
	 */
	private class CapCvTask implements Runnable {

		private final Log log = LogFactory.getLog(Spider.class);
		private String companyId = null;
		private Connection conn = null;
		private Gson gson = null;
		private QueryRunner queryRunner = null;

		private int RETRY_TIMES = 10;

		private Set<String> keySet = null;

		/**
		 * 公司ID
		 * 
		 * @param companyId
		 */
		public CapCvTask(String companyId) {
			this.companyId = companyId;
			gson = new Gson();
			queryRunner = new QueryRunner();
			keySet = new TreeSet<String>();
			try {
				conn = DBManager.getConnection();
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}

		@Override
		public void run() {
			String url = "http://www.ipin.com/company/getCompanyDetails.do";
			String sql = "insert into SP_COMPANY_V (CV_ID, BAK_ID, AGE, GENDER, DIPLOMA, EDU_START, EDU_END, EDU_MAJOR, EDU_MAJOR_ID, JOB_START, JOB_INC, JOB_INC_ID, JOB_IND, JOB_LOC, JOB_POSITION, JOB_CATE, JOB_SALARY, JOB_INDEX, JOB_EXP, INC_RANK, INC_RANK_WIDTH, SCH_RANK, SCH_RANK_WIDTH, TAG, SHOW_INC_NAME, EDU_SCH, EDU_SCH_ID, OLD_DATA)"
					+ " values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

			String queryKeySetSql = "select t.cv_id from sp_company_v t where t.job_inc_id = ?";
			try {
				queryRunner
						.update(conn, "update sp_company t set t.cv_status = 1 where t.sp_company_id = ?", companyId); // 更新公司状态为1.进行中
				List<Object[]> rsList = queryRunner.query(conn, queryKeySetSql, new ArrayListHandler(), companyId);
				for (Object[] key : rsList) {
					keySet.add((String) key[0]);
				}
			} catch (SQLException e1) {
				e1.printStackTrace();
			}
			int retryCount = RETRY_TIMES;
			String iPinUserId = getIpinUserId(false);
			while (true) {
				int count = 0;
				try {
					String docString = Jsoup.connect(url).cookie("ipinUserId", iPinUserId).data("inc_id", companyId)
							.execute().body();
					if (!docString.contains("用以下账号登录，即可免费查看公司招怎样的人及拿多少钱等信息")) {
						String findLabel = "\"json\": {\"runType\": \"incPosition\", \"cvList\":";
						int begin = docString.indexOf(findLabel);
						int end = docString.indexOf("}, \"tpl\": \"similar\",", begin);
						docString = docString.substring(begin + findLabel.length(), end);
						List<Map<String, Object>> cvMapList = gson.fromJson(docString, List.class);
						for (Map<String, Object> cvMap : cvMapList) {
							if (!keySet.contains(cvMap.get("cvId"))) {
								count++;
								queryRunner.update(conn, sql, String.valueOf(cvMap.get("cvId")),
										String.valueOf(cvMap.get("_id")), String.valueOf(cvMap.get("age")),
										String.valueOf(cvMap.get("gender")), String.valueOf(cvMap.get("diploma")),
										String.valueOf(cvMap.get("eduStart")), String.valueOf(cvMap.get("eduEnd")),
										String.valueOf(cvMap.get("eduMajor")), String.valueOf(cvMap.get("eduMajorId")),
										String.valueOf(cvMap.get("jobStart")), String.valueOf(cvMap.get("jobInc")),
										String.valueOf(cvMap.get("jobIncId")), String.valueOf(cvMap.get("jobInd")),
										String.valueOf(cvMap.get("jobLoc")), String.valueOf(cvMap.get("jobPosition")),
										String.valueOf(cvMap.get("jobCate")), String.valueOf(cvMap.get("jobSalary")),
										String.valueOf(cvMap.get("jobIndex")), String.valueOf(cvMap.get("jobExp")),
										String.valueOf(cvMap.get("inc_rank")),
										String.valueOf(cvMap.get("inc_rank_width")),
										String.valueOf(cvMap.get("sch_rank")),
										String.valueOf(cvMap.get("sch_rank_width")), String.valueOf(cvMap.get("tag")),
										String.valueOf(cvMap.get("showIncName")), String.valueOf(cvMap.get("eduSch")),
										String.valueOf(cvMap.get("eduSchId")), String.valueOf(cvMap.get("oldData")));
								keySet.add((String) cvMap.get("cvId"));
							}
						}
						log.info("公司ID：" + companyId + " 获得" + cvMapList.size() + "条记录，插入" + count + "条记录。");
					} else {
						log.info("账号" + iPinUserId + "今日配额使用完毕，更换下一个账号抓取！");
						iPinUserId = getIpinUserId(false);
						if (iPinUserId == null) {
							log.info("账号资源用尽，请明日继续抓取");
							queryRunner.update(conn, "update sp_user t set t.state = 9 where t.ipinuserid = ?",
									iPinUserId); // 更新用户状态为9禁用
							break;
						}
						log.info("开始使用账号" + iPinUserId + "进行抓取！");
					}
				} catch (IOException e) {
					e.printStackTrace();
				} catch (SQLException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				if (count == 0) {
					if ((retryCount--) <= 0) {
						break;
					}
				} else {
					retryCount = RETRY_TIMES;
				}
			}

			try {
				queryRunner.update(conn, "update sp_user t set t.state = 0,t.count = t.count+1 where t.ipinuserid = ?",
						iPinUserId); // 更新用户状态为0正常
				queryRunner
						.update(conn, "update sp_company t set t.cv_status = 9 where t.sp_company_id = ?", companyId);
				log.info("------公司" + companyId + "CV数据抓取完成！-------");
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	public static void main(String[] args) {
		Spider spider = new Spider();
		// spider.capCompany(); //抓取学校列表
		spider.initUser(); // 初始化用户
		// spider.capCompanyDetail(); //抓取公司详情
		// spider.analysisCompanyDetail();
		// spider.capCvList();
		spider.capCvListMultiThread(10);
	}
}
